import pandas as pd
import datetime
from config import config
from utils.read_url import get_url


def read_page_info(page_num):
    url = config.URL.format(page_num)
    req = get_url(url)
    if req:
        req_data = req.get('Table')
        return req_data


def read_main():
    arr = []
    i = 1
    while True:
        mid_arr = read_page_info(page_num=i)
        if mid_arr:
            arr += mid_arr
            i += 1
        else:
            break

    df = pd.DataFrame(arr)
    return df.sort_values('ProdProfit', ascending=False)


def read_main_today():
    arr = []
    i = 1
    while True:
        mid_arr = read_page_info(page_num=i)
        if mid_arr:
            arr += mid_arr
            i += 1
        else:
            break

    df = pd.DataFrame(arr)
    if arr:
        today = datetime.datetime.now().strftime('%Y.%m.%d')
        df = pd.DataFrame(arr)
        df = df[df['szComDat'] == today]
        if not df.empty:
            df.sort_values('ProdProfit', ascending=False)
    return df


# df = read_main()
df = read_main_today()
print(df)
print(len(df))